diff --git a/go.mod b/go.mod index 3c9c9738f50..975c23af596 100644 --- a/go.mod +++ b/go.mod @@ -60,9 +60,10 @@ require ( github.com/yuin/goldmark-highlighting v0.0.0-20200307114337-60d527fdb691 gocloud.dev v0.20.0 golang.org/x/image v0.0.0-20211028202545-6944b10bf410 - golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d + golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/text v0.3.7 + golang.org/x/tools v0.1.9 // indirect google.golang.org/api v0.63.0 gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index 6373c1c62af..a2bdd4ae0f0 100644 --- a/go.sum +++ b/go.sum @@ -619,6 +619,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.4 h1:zNWRjYUW32G9KirMXYHQHVNFkXvMI7LpgNW2AgYAoIs= github.com/yuin/goldmark v1.4.4/go.mod h1:rmuwmfZ0+bvzB24eSC//bk1R1Zp3hM0OXYv/G2LIilg= github.com/yuin/goldmark-highlighting v0.0.0-20200307114337-60d527fdb691 h1:VWSxtAiQNh3zgHJpdpkpVYjTPqRE3P6UZCOPa1nRDio= @@ -695,6 +696,7 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -744,6 +746,8 @@ golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -845,6 +849,7 @@ golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486 h1:5hpz5aRr+W1erYCL5JRhSUBJRph7l9XkNveoExlrKYk= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -924,6 +929,8 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.9 h1:j9KsMiaP1c3B0OTQGth0/k+miLGTgLsAFUCrF2vLcF8= +golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index 91703091bb5..f1930fd7155 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -117,6 +117,7 @@ func (h *HugoSites) getContentMaps() *pageMaps { // Only used in tests. type testCounters struct { contentRenderCounter uint64 + pageRenderCounter uint64 } func (h *testCounters) IncrContentRender() { @@ -126,6 +127,13 @@ func (h *testCounters) IncrContentRender() { atomic.AddUint64(&h.contentRenderCounter, 1) } +func (h *testCounters) IncrPageRender() { + if h == nil { + return + } + atomic.AddUint64(&h.pageRenderCounter, 1) +} + type fatalErrorHandler struct { mu sync.Mutex diff --git a/hugolib/integrationtest_builder.go b/hugolib/integrationtest_builder.go new file mode 100644 index 00000000000..7ec7a150325 --- /dev/null +++ b/hugolib/integrationtest_builder.go @@ -0,0 +1,448 @@ +package hugolib + +import ( + "bytes" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + jww "github.com/spf13/jwalterweatherman" + + qt "github.com/frankban/quicktest" + "github.com/fsnotify/fsnotify" + "github.com/gohugoio/hugo/common/herrors" + "github.com/gohugoio/hugo/common/hexec" + "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/config/security" + "github.com/gohugoio/hugo/deps" + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/htesting" + "github.com/gohugoio/hugo/hugofs" + "github.com/spf13/afero" + "golang.org/x/tools/txtar" +) + +func NewIntegrationTestBuilder(conf IntegrationTestConfig) *IntegrationTestBuilder { + data := txtar.Parse([]byte(conf.TxtarString)) + + c, ok := conf.T.(*qt.C) + if !ok { + c = qt.New(conf.T) + } + + if conf.NeedsOsFS { + doClean := true + tempDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-integration-test") + c.Assert(err, qt.IsNil) + conf.WorkingDir = filepath.Join(tempDir, conf.WorkingDir) + if doClean { + c.Cleanup(clean) + } + } + + return &IntegrationTestBuilder{ + Cfg: conf, + C: c, + data: data, + } +} + +// IntegrationTestBuilder is a (partial) rewrite of sitesBuilder. +// The main problem with the "old" one was that it was that the test data was often a little hidden, +// so it became hard to look at a test and determine what it should do, especially coming back to the +// test after a year or so. +type IntegrationTestBuilder struct { + *qt.C + + data *txtar.Archive + + fs *hugofs.Fs + H *HugoSites + + Cfg IntegrationTestConfig + + changedFiles []string + createdFiles []string + removedFiles []string + renamedFiles []string + + buildCount int + counters *testCounters + logBuff lockingBuffer + + builderInit sync.Once +} + +type lockingBuffer struct { + sync.Mutex + bytes.Buffer +} + +func (b *lockingBuffer) Write(p []byte) (n int, err error) { + b.Lock() + n, err = b.Buffer.Write(p) + b.Unlock() + return +} + +func (s *IntegrationTestBuilder) AssertLogContains(text string) { + s.Helper() + s.Assert(s.logBuff.String(), qt.Contains, text) +} + +func (s *IntegrationTestBuilder) AssertBuildCountData(count int) { + s.Helper() + s.Assert(s.H.init.data.InitCount(), qt.Equals, count) +} + +func (s *IntegrationTestBuilder) AssertBuildCountGitInfo(count int) { + s.Helper() + s.Assert(s.H.init.gitInfo.InitCount(), qt.Equals, count) +} + +func (s *IntegrationTestBuilder) AssertBuildCountLayouts(count int) { + s.Helper() + s.Assert(s.H.init.layouts.InitCount(), qt.Equals, count) +} + +func (s *IntegrationTestBuilder) AssertBuildCountTranslations(count int) { + s.Helper() + s.Assert(s.H.init.translations.InitCount(), qt.Equals, count) +} + +func (s *IntegrationTestBuilder) AssertFileContent(filename string, matches ...string) { + s.Helper() + content := strings.TrimSpace(s.FileContent(filename)) + for _, m := range matches { + lines := strings.Split(m, "\n") + for _, match := range lines { + match = strings.TrimSpace(match) + if match == "" || strings.HasPrefix(match, "#") { + continue + } + s.Assert(content, qt.Contains, match, qt.Commentf(content)) + } + } +} + +func (s *IntegrationTestBuilder) AssertDestinationExists(filename string, b bool) { + checker := qt.IsTrue + if !b { + checker = qt.IsFalse + } + s.Assert(s.destinationExists(filepath.Clean(filename)), checker) +} + +func (s *IntegrationTestBuilder) destinationExists(filename string) bool { + b, err := helpers.Exists(filename, s.fs.Destination) + if err != nil { + panic(err) + } + return b +} + +func (s *IntegrationTestBuilder) AssertIsFileError(err error) { + var ferr *herrors.ErrorWithFileContext + s.Assert(err, qt.ErrorAs, &ferr) +} + +func (s *IntegrationTestBuilder) AssertRenderCountContent(count int) { + s.Helper() + s.Assert(s.counters.contentRenderCounter, qt.Equals, uint64(count)) +} + +func (s *IntegrationTestBuilder) AssertRenderCountPage(count int) { + s.Helper() + s.Assert(s.counters.pageRenderCounter, qt.Equals, uint64(count)) +} + +func (s *IntegrationTestBuilder) Build() *IntegrationTestBuilder { + s.Helper() + _, err := s.BuildE() + if s.Cfg.Verbose { + fmt.Println(s.logBuff.String()) + } + s.Assert(err, qt.IsNil) + return s +} + +func (s *IntegrationTestBuilder) BuildE() (*IntegrationTestBuilder, error) { + s.Helper() + s.initBuilder() + err := s.build(BuildCfg{}) + return s, err +} + +type IntegrationTestDebugConfig struct { + Out io.Writer + + PrintDestinationFs bool + PrintPagemap bool + + PrefixDestinationFs string + PrefixPagemap string +} + +func (s *IntegrationTestBuilder) EditFileReplace(filename string, replacementFunc func(s string) string) *IntegrationTestBuilder { + absFilename := s.absFilename(filename) + b, err := afero.ReadFile(s.fs.Source, absFilename) + s.Assert(err, qt.IsNil) + s.changedFiles = append(s.changedFiles, absFilename) + oldContent := string(b) + s.writeSource(absFilename, replacementFunc(oldContent)) + return s +} + +func (s *IntegrationTestBuilder) EditFiles(filenameContent ...string) *IntegrationTestBuilder { + for i := 0; i < len(filenameContent); i += 2 { + filename, content := filepath.FromSlash(filenameContent[i]), filenameContent[i+1] + absFilename := s.absFilename(filename) + s.changedFiles = append(s.changedFiles, absFilename) + s.writeSource(absFilename, content) + } + return s +} + +func (s *IntegrationTestBuilder) AddFiles(filenameContent ...string) *IntegrationTestBuilder { + for i := 0; i < len(filenameContent); i += 2 { + filename, content := filepath.FromSlash(filenameContent[i]), filenameContent[i+1] + absFilename := s.absFilename(filename) + s.createdFiles = append(s.createdFiles, absFilename) + s.writeSource(absFilename, content) + } + return s +} + +func (s *IntegrationTestBuilder) RemoveFiles(filenames ...string) *IntegrationTestBuilder { + for _, filename := range filenames { + absFilename := s.absFilename(filename) + s.removedFiles = append(s.removedFiles, absFilename) + s.Assert(s.fs.Source.Remove(absFilename), qt.IsNil) + + } + + return s +} + +func (s *IntegrationTestBuilder) RenameFile(old, new string) *IntegrationTestBuilder { + absOldFilename := s.absFilename(old) + absNewFilename := s.absFilename(new) + s.renamedFiles = append(s.renamedFiles, absOldFilename) + s.createdFiles = append(s.createdFiles, absNewFilename) + s.Assert(s.fs.Source.Rename(absOldFilename, absNewFilename), qt.IsNil) + return s +} + +func (s *IntegrationTestBuilder) FileContent(filename string) string { + s.Helper() + filename = filepath.FromSlash(filename) + if !strings.HasPrefix(filename, s.Cfg.WorkingDir) { + filename = filepath.Join(s.Cfg.WorkingDir, filename) + } + return s.readDestination(s, s.fs, filename) +} + +func (s *IntegrationTestBuilder) initBuilder() { + s.builderInit.Do(func() { + var afs afero.Fs + if s.Cfg.NeedsOsFS { + afs = afero.NewOsFs() + } else { + afs = afero.NewMemMapFs() + } + + if s.Cfg.LogLevel == 0 { + s.Cfg.LogLevel = jww.LevelWarn + } + + logger := loggers.NewBasicLoggerForWriter(s.Cfg.LogLevel, &s.logBuff) + + fs := hugofs.NewFrom(afs, config.New()) + + for _, f := range s.data.Files { + filename := filepath.Join(s.Cfg.WorkingDir, f.Name) + s.Assert(afs.MkdirAll(filepath.Dir(filename), 0777), qt.IsNil) + s.Assert(afero.WriteFile(afs, filename, bytes.TrimSuffix(f.Data, []byte("\n")), 0666), qt.IsNil) + } + + cfg, _, err := LoadConfig( + ConfigSourceDescriptor{ + WorkingDir: s.Cfg.WorkingDir, + Fs: afs, + Logger: logger, + Environ: []string{}, + Filename: "config.toml", + }, + func(cfg config.Provider) error { + return nil + }, + ) + + s.Assert(err, qt.IsNil) + + cfg.Set("workingDir", s.Cfg.WorkingDir) + + depsCfg := deps.DepsCfg{Cfg: cfg, Fs: fs, Running: s.Cfg.Running, Logger: logger} + sites, err := NewHugoSites(depsCfg) + s.Assert(err, qt.IsNil) + + s.H = sites + s.fs = fs + + if s.Cfg.NeedsNpmInstall { + wd, _ := os.Getwd() + s.Assert(os.Chdir(s.Cfg.WorkingDir), qt.IsNil) + s.C.Cleanup(func() { os.Chdir(wd) }) + sc := security.DefaultConfig + sc.Exec.Allow = security.NewWhitelist("npm") + ex := hexec.New(sc) + command, err := ex.New("npm", "install") + s.Assert(err, qt.IsNil) + s.Assert(command.Run(), qt.IsNil) + + } + }) +} + +func (s *IntegrationTestBuilder) absFilename(filename string) string { + filename = filepath.FromSlash(filename) + if filepath.IsAbs(filename) { + return filename + } + if s.Cfg.WorkingDir != "" && !strings.HasPrefix(filename, s.Cfg.WorkingDir) { + filename = filepath.Join(s.Cfg.WorkingDir, filename) + } + return filename +} + +func (s *IntegrationTestBuilder) build(cfg BuildCfg) error { + s.Helper() + defer func() { + s.changedFiles = nil + s.createdFiles = nil + s.removedFiles = nil + s.renamedFiles = nil + }() + + changeEvents := s.changeEvents() + s.logBuff.Reset() + s.counters = &testCounters{} + cfg.testCounters = s.counters + + if s.buildCount > 0 && (len(changeEvents) == 0) { + return nil + } + + s.buildCount++ + + err := s.H.Build(cfg, changeEvents...) + if err != nil { + return err + } + logErrorCount := s.H.NumLogErrors() + if logErrorCount > 0 { + return fmt.Errorf("logged %d error(s): %s", logErrorCount, s.logBuff.String()) + } + + return nil +} + +func (s *IntegrationTestBuilder) changeEvents() []fsnotify.Event { + var events []fsnotify.Event + for _, v := range s.removedFiles { + events = append(events, fsnotify.Event{ + Name: v, + Op: fsnotify.Remove, + }) + } + for _, v := range s.renamedFiles { + events = append(events, fsnotify.Event{ + Name: v, + Op: fsnotify.Rename, + }) + } + for _, v := range s.changedFiles { + events = append(events, fsnotify.Event{ + Name: v, + Op: fsnotify.Write, + }) + } + for _, v := range s.createdFiles { + events = append(events, fsnotify.Event{ + Name: v, + Op: fsnotify.Create, + }) + } + + return events +} + +func (s *IntegrationTestBuilder) readDestination(t testing.TB, fs *hugofs.Fs, filename string) string { + t.Helper() + return s.readFileFromFs(t, fs.Destination, filename) +} + +func (s *IntegrationTestBuilder) readFileFromFs(t testing.TB, fs afero.Fs, filename string) string { + t.Helper() + filename = filepath.Clean(filename) + b, err := afero.ReadFile(fs, filename) + if err != nil { + // Print some debug info + hadSlash := strings.HasPrefix(filename, helpers.FilePathSeparator) + start := 0 + if hadSlash { + start = 1 + } + end := start + 1 + + parts := strings.Split(filename, helpers.FilePathSeparator) + if parts[start] == "work" { + end++ + } + + s.Assert(err, qt.IsNil) + + } + return string(b) +} + +func (s *IntegrationTestBuilder) writeSource(filename, content string) { + s.Helper() + s.writeToFs(s.fs.Source, filename, content) +} + +func (s *IntegrationTestBuilder) writeToFs(fs afero.Fs, filename, content string) { + s.Helper() + if err := afero.WriteFile(fs, filepath.FromSlash(filename), []byte(content), 0755); err != nil { + s.Fatalf("Failed to write file: %s", err) + } +} + +type IntegrationTestConfig struct { + T testing.TB + + // The files to use on txtar format, see + // https://pkg.go.dev/golang.org/x/exp/cmd/txtar + TxtarString string + + // Whether to simulate server mode. + Running bool + + // Will print the log buffer after the build + Verbose bool + + LogLevel jww.Threshold + + // Whether it needs the real file system (e.g. for js.Build tests). + NeedsOsFS bool + + // Whether to run npm install before Build. + NeedsNpmInstall bool + + WorkingDir string +} diff --git a/hugolib/site.go b/hugolib/site.go index 13d5482b1ef..02380a6e73c 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -1719,6 +1719,7 @@ func (s *Site) renderAndWriteXML(statCounter *uint64, name string, targetPath st func (s *Site) renderAndWritePage(statCounter *uint64, name string, targetPath string, p *pageState, templ tpl.Template) error { s.Log.Debugf("Render %s to %q", name, targetPath) + s.h.IncrPageRender() renderBuffer := bp.GetBuffer() defer bp.PutBuffer(renderBuffer) diff --git a/lazy/init.go b/lazy/init.go index 6dff0c98c29..fc64b2a7da3 100644 --- a/lazy/init.go +++ b/lazy/init.go @@ -16,6 +16,7 @@ package lazy import ( "context" "sync" + "sync/atomic" "time" "github.com/pkg/errors" @@ -28,6 +29,9 @@ func New() *Init { // Init holds a graph of lazily initialized dependencies. type Init struct { + // Used in tests + initCount uint64 + mu sync.Mutex prev *Init @@ -47,6 +51,12 @@ func (ini *Init) Add(initFn func() (interface{}, error)) *Init { return ini.add(false, initFn) } +// InitCount gets the number of this this Init has been initialized. +func (ini *Init) InitCount() int { + i := atomic.LoadUint64(&ini.initCount) + return int(i) +} + // AddWithTimeout is same as Add, but with a timeout that aborts initialization. func (ini *Init) AddWithTimeout(timeout time.Duration, f func(ctx context.Context) (interface{}, error)) *Init { return ini.Add(func() (interface{}, error) { @@ -77,6 +87,7 @@ func (ini *Init) Do() (interface{}, error) { } ini.init.Do(func() { + atomic.AddUint64(&ini.initCount, 1) prev := ini.prev if prev != nil { // A branch. Initialize the ancestors.