Merge branch 'main' into dev-stream-change-max-size

.github/dependabot.yml

@@ -0,0 +1,22 @@
+version: 2
+updates:
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "wednesday"
+      time: "16:00"
+    rebase-strategy: "disabled"
+    groups:
+      patch:
+        update-types: ["patch"]
+      minor:
+        update-types: ["minor"]
+
+  - package-ecosystem: "github-actions"
+    # Workflow files stored in the default location of `.github/workflows`. (You don't need to specify `/.github/workflows` for `directory`. You can use `directory: "/"`.)
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: wednesday
+      time: "16:00"

.github/workflows/cd-badger.yml

@@ -45,7 +45,7 @@ jobs:
       - name: Tar Archive for Linux Build
         run: cd badger && tar -zcvf badger-linux-amd64.tar.gz badger-linux-amd64
       - name: Upload Badger Binary Build Artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           path: |
             badger/badger-checksum-linux-amd64.sha256
@@ -88,7 +88,7 @@ jobs:
       - name: Tar Archive for Linux Build
         run: cd badger && tar -zcvf badger-linux-arm64.tar.gz badger-linux-arm64
       - name: Upload Badger Binary Build Artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           path: |
             badger/badger-checksum-linux-arm64.sha256

.github/workflows/ci-aqua-security-trivy-tests.yml

@@ -8,6 +8,12 @@ on:
       - ready_for_review
     branches:
       - main
+  schedule:
+    - cron: "0 0 * * *"
+
+permissions:
+  security-events: write
+
 jobs:
   build:
     name: trivy-tests
@@ -23,6 +29,6 @@ jobs:
           format: 'sarif'
           output: 'trivy-results.sarif'
       - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v2
+        uses: github/codeql-action/upload-sarif@v3
         with:
           sarif_file: 'trivy-results.sarif'

.github/workflows/ci-golang-lint.yml

@@ -20,11 +20,10 @@ jobs:
         with:
           go-version: ${{ env.GOVERSION }}
       - name: golang-lint
-        uses: golangci/golangci-lint-action@v4
+        uses: golangci/golangci-lint-action@v6
         with:
           # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
           version: latest
           only-new-issues: true
           args: --timeout=10m
-          skip-pkg-cache: true
-          skip-build-cache: true
+          skip-cache: true

.github/workflows/stale.yml

@@ -0,0 +1,18 @@
+name: 'Close stale issues and PRs'
+on:
+  schedule:
+    - cron: '00 02,14 * * *'
+
+permissions:
+  issues: write
+  pull-requests: write
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v9
+        with:
+          stale-issue-message: 'This issue has been stale for 60 days and will be closed automatically in 7 days. Comment to keep it open.'
+          stale-pr-message: 'This PR has been stale for 60 days and will be closed automatically in 7 days. Comment to keep it open.'
+          operations-per-run: 100

README.md

@@ -221,6 +221,7 @@ Below is a list of known projects that use Badger:
 * [DVID](https://github.com/janelia-flyem/dvid) - A dataservice for branched versioning of a variety of data types. Originally created for large-scale brain reconstructions in Connectomics.
 * [KVS](https://github.com/tauraamui/kvs) - A library for making it easy to persist, load and query full structs into BadgerDB, using an ownership hierarchy model.
 * [LLS](https://github.com/Boc-chi-no/LLS) - LLS is an efficient URL Shortener that can be used to shorten links and track link usage. Support for BadgerDB and MongoDB. Improved performance by more than 30% when using BadgerDB
+* [lakeFS](https://github.com/treeverse/lakeFS) - lakeFS is an open-source data version control that transforms your object storage to Git-like repositories. lakeFS uses BadgerDB for its underlying local metadata KV store implementation.
 
 If you are using Badger in a project please send a pull request to add it to the list.
 

db.go

@@ -34,6 +34,7 @@ import (
 	humanize "github.com/dustin/go-humanize"
 	"github.com/pkg/errors"
 
+	"github.com/dgraph-io/badger/v4/fb"
 	"github.com/dgraph-io/badger/v4/options"
 	"github.com/dgraph-io/badger/v4/pb"
 	"github.com/dgraph-io/badger/v4/skl"
@@ -123,8 +124,8 @@ type DB struct {
 
 	pub        *publisher
 	registry   *KeyRegistry
-	blockCache *ristretto.Cache
-	indexCache *ristretto.Cache
+	blockCache *ristretto.Cache[[]byte, *table.Block]
+	indexCache *ristretto.Cache[uint64, *fb.TableIndex]
 	allocPool  *z.AllocatorPool
 }
 
@@ -165,7 +166,7 @@ func checkAndSetOptions(opt *Options) error {
 			"reduce opt.ValueThreshold or increase opt.BaseTableSize.",
 			opt.ValueThreshold, opt.maxBatchSize)
 	}
-	// ValueLogFileSize should be stricly LESS than 2<<30 otherwise we will
+	// ValueLogFileSize should be strictly LESS than 2<<30 otherwise we will
 	// overflow the uint32 when we mmap it in OpenMemtable.
 	if !(opt.ValueLogFileSize < 2<<30 && opt.ValueLogFileSize >= 1<<20) {
 		return ErrValueLogSize
@@ -274,14 +275,14 @@ func Open(opt Options) (*DB, error) {
 			numInCache = 1
 		}
 
-		config := ristretto.Config{
+		config := ristretto.Config[[]byte, *table.Block]{
 			NumCounters: numInCache * 8,
 			MaxCost:     opt.BlockCacheSize,
 			BufferItems: 64,
 			Metrics:     true,
 			OnExit:      table.BlockEvictHandler,
 		}
-		db.blockCache, err = ristretto.NewCache(&config)
+		db.blockCache, err = ristretto.NewCache[[]byte, *table.Block](&config)
 		if err != nil {
 			return nil, y.Wrap(err, "failed to create data cache")
 		}
@@ -297,7 +298,7 @@ func Open(opt Options) (*DB, error) {
 			numInCache = 1
 		}
 
-		config := ristretto.Config{
+		config := ristretto.Config[uint64, *fb.TableIndex]{
 			NumCounters: numInCache * 8,
 			MaxCost:     opt.IndexCacheSize,
 			BufferItems: 64,
@@ -402,7 +403,7 @@ func Open(opt Options) (*DB, error) {
 	return db, nil
 }
 
-// initBannedNamespaces retrieves the banned namepsaces from the DB and updates in-memory structure.
+// initBannedNamespaces retrieves the banned namespaces from the DB and updates in-memory structure.
 func (db *DB) initBannedNamespaces() error {
 	if db.opt.NamespaceOffset < 0 {
 		return nil
@@ -904,7 +905,7 @@ func (db *DB) sendToWriteCh(entries []*Entry) (*request, error) {
 		return nil, ErrTxnTooBig
 	}
 
-	// We can only service one request because we need each txn to be stored in a contigous section.
+	// We can only service one request because we need each txn to be stored in a contiguous section.
 	// Txns should not interleave among other txns or rewrites.
 	req := requestPool.Get().(*request)
 	req.reset()
@@ -1709,7 +1710,7 @@ func (db *DB) dropAll() (func(), error) {
 	if err != nil {
 		return f, err
 	}
-	// prepareToDrop will stop all the incomming write and flushes any pending memtables.
+	// prepareToDrop will stop all the incoming write and flushes any pending memtables.
 	// Before we drop, we'll stop the compaction because anyways all the datas are going to
 	// be deleted.
 	db.stopCompactions()
@@ -1752,7 +1753,7 @@ func (db *DB) dropAll() (func(), error) {
 
 // DropPrefix would drop all the keys with the provided prefix. It does this in the following way:
 //   - Stop accepting new writes.
-//   - Stop memtable flushes before acquiring lock. Because we're acquring lock here
+//   - Stop memtable flushes before acquiring lock. Because we're acquiring lock here
 //     and memtable flush stalls for lock, which leads to deadlock
 //   - Flush out all memtables, skipping over keys with the given prefix, Kp.
 //   - Write out the value log header to memtables when flushing, so we don't accidentally bring Kp

docs/config.toml

@@ -1,7 +1,6 @@
 languageCode = "en-us"
 theme = "hugo-docs"
 canonifyURLs = false
-relativeURLs = true
 
 [markup.goldmark.renderer]
 unsafe = true

docs/content/projects-using-badger/index.md

@@ -58,5 +58,6 @@ Below is a list of known projects that use Badger:
 * [DVID](https://github.com/janelia-flyem/dvid) - A dataservice for branched versioning of a variety of data types. Originally created for large-scale brain reconstructions in Connectomics.
 * [KVS](https://github.com/tauraamui/kvs) - A library for making it easy to persist, load and query full structs into BadgerDB, using an ownership hierarchy model.
 * [LLS](https://github.com/Boc-chi-no/LLS) - LLS is an efficient URL Shortener that can be used to shorten links and track link usage. Support for BadgerDB and MongoDB. Improved performance by more than 30% when using BadgerDB
+* [ActionManager](https://mftlabs.io/actionmanager) - A dynamic entity manager based on rjsf schema and badger db
 
 If you are using Badger in a project please send a pull request to add it to the list.

docs/themes/hugo-docs/layouts/partials/header.html

@@ -29,7 +29,7 @@
     <script src="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.10.0/highlight.min.js"></script>
     <script async src="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.10.0/languages/go.min.js"></script>
     <script async src="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.10.0/languages/java.min.js" integrity="sha256-RyOs5MjQ+Ef9DQrPHp4QGr2tcNjxLh6mCkkW+sS4doU=" crossorigin="anonymous"></script>
-    <script src="//code.jquery.com/jquery-3.2.1.min.js" integrity="sha256-hwg4gsxgFZhOsEEamdOYGBf13FyQuiTwlAQgxVSNgt4=" crossorigin="anonymous"></script>
+    <script src="//code.jquery.com/jquery-3.7.1.min.js" integrity="sha256-/JqT3SQfawRcv/BIHPThkBvs0OEvtFFmqPF/lYI/Cxo=" crossorigin="anonymous"></script>
 
     <title>
       {{.Section | default "Badgerdb Documentation" | humanize}} — {{ .Site.Title }}
@@ -44,7 +44,7 @@
     <script src="//maxcdn.bootstrapcdn.com/bootstrap/4.0.0-alpha.6/js/bootstrap.min.js" integrity="sha384-vBWWzlZJ8ea9aCX4pEW3rVHjgjt7zpkNpZk+02D9phzyeVkE+jo0ieGizqPLForn" crossorigin="anonymous"></script>
     <link href="//fonts.googleapis.com/css?family=Lato:400,700" rel="stylesheet" />
     <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.7.0/css/font-awesome.min.css" rel="stylesheet" integrity="sha384-wvfXpqpZZVQGK6TAh5PVlGOfQNHSoD2xbE+QkPxCAFlNEevoEH3Sl0sibVcOQVnN" crossorigin="anonymous" />
-    <link href="https://cdn.lineicons.com/2.0/LineIcons.css" rel="stylesheet" crossorigin="anonymous" />
+    <link href="//cdn.lineicons.com/2.0/LineIcons.css" rel="stylesheet" crossorigin="anonymous" />
 
     <!-- DocSearch -->
     <link rel="stylesheet" href="//cdn.jsdelivr.net/npm/docsearch.js@2/dist/cdn/docsearch.min.css" />

go.mod

@@ -3,32 +3,31 @@ module github.com/dgraph-io/badger/v4
 go 1.19
 
 require (
-	github.com/cespare/xxhash/v2 v2.2.0
-	github.com/dgraph-io/ristretto v0.1.1
-	github.com/dustin/go-humanize v1.0.0
+	github.com/cespare/xxhash/v2 v2.3.0
+	github.com/dgraph-io/ristretto v0.1.2-0.20240116140435-c67e07994f91
+	github.com/dustin/go-humanize v1.0.1
 	github.com/gogo/protobuf v1.3.2
-	github.com/golang/protobuf v1.5.2
+	github.com/golang/protobuf v1.5.4
 	github.com/google/flatbuffers v1.12.1
-	github.com/klauspost/compress v1.15.15
+	github.com/klauspost/compress v1.17.9
 	github.com/pkg/errors v0.9.1
-	github.com/spf13/cobra v1.7.0
-	github.com/stretchr/testify v1.4.0
-	go.opencensus.io v0.22.5
-	golang.org/x/net v0.17.0
-	golang.org/x/sys v0.13.0
+	github.com/spf13/cobra v1.8.1
+	github.com/stretchr/testify v1.9.0
+	go.opencensus.io v0.24.0
+	golang.org/x/net v0.28.0
+	golang.org/x/sys v0.24.0
 )
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
-	github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb // indirect
-	google.golang.org/grpc v1.20.1 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
-	gopkg.in/yaml.v2 v2.2.2 // indirect
+	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
+	google.golang.org/grpc v1.56.3 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
 retract v4.0.0 // see #1888 and #1889